{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "34b3fcc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import math\n",
    "from scipy import stats\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "\n",
    "pd.set_option('display.max_rows', 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "8a5c9b03",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Results on the first 8 methods\n",
    "# results = pd.read_csv('mturk/sc09-unconditional-exp-confident/Batch_4656948_batch_results.csv')\n",
    "\n",
    "# uids = []\n",
    "# for method in [\n",
    "#     'mturk/sc09-unconditional-exp-confident-diffwave-1m/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-diffwave-500k/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-samplernn-3/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-sashimi-8-glu/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-sashimi-diffwave-500k/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-test/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-wavegan/',\n",
    "#     'mturk/sc09-unconditional-exp-confident-wavenet-1024/',\n",
    "# ]:\n",
    "#     uids.append(pd.read_csv(\n",
    "#         f'{method}/uids.txt', \n",
    "#         sep=' ', \n",
    "#         header=None, \n",
    "#         names=['method', 'filename','uid'],\n",
    "#     ))\n",
    "    \n",
    "# uids = pd.concat(uids, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "cfadde0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results on the next 4 methods\n",
    "results = pd.read_csv('Batch_4657899_batch_results.csv')\n",
    "\n",
    "uids = []\n",
    "for method in [\n",
    "    '../final/sc09-unconditional-exp-confident-diffwave-small-500k/',\n",
    "    '../final/sc09-unconditional-exp-confident-sashimi-diffwave-small-500k/',\n",
    "    '../final/sc09-unconditional-exp-confident-sashimi-diffwave-800k/',\n",
    "    '../final/sc09-unconditional-exp-confident-sashimi-diffwave-snet-uni-500k/',\n",
    "    # replace or add your methods\n",
    "]:\n",
    "    uids.append(pd.read_csv(\n",
    "        f'{method}/uids.txt', \n",
    "        sep=' ', \n",
    "        header=None, \n",
    "        names=['method', 'filename','uid'],\n",
    "    ))\n",
    "    \n",
    "uids = pd.concat(uids, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "71abd2be",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = []\n",
    "for col in results.columns:\n",
    "    if col.startswith('Input'):\n",
    "        index = int(col.split(\"_\")[-2])\n",
    "        responses.append(\n",
    "            results[\n",
    "                [col, f'Answer.recording_{index}_intelligibility', f'Answer.recording_{index}_digit', 'WorkerId', 'Answer.diversity', 'Answer.quality']\n",
    "            ]\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "ecaa27ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "responses = [e.rename(columns=dict(zip(e.columns, ['url', 'intelligibility', 'digit', 'worker', 'diversity', 'quality']))) for e in responses]\n",
    "responses = pd.concat(responses, axis=0)\n",
    "responses['uid'] = responses['url'].apply(lambda x: x.split(\"/\")[-1].replace(\".wav\", \"\"))\n",
    "responses['method'] = responses['url'].apply(lambda x: x.split(\"/\")[-3].replace(\"sc09-unconditional-exp-confident-\", \"\"))\n",
    "responses = responses[['uid', 'method', 'worker', 'digit', 'intelligibility', 'quality', 'diversity']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "268e6a7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.merge(responses, uids, on=['uid', 'method'], how='inner')\n",
    "data = data.sort_values(['filename', 'method']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "99890edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filenames are sorted by digit class (e.g. 0.wav, ..., 49.wav are digit zero and so on)\n",
    "classes = pd.DataFrame({\n",
    "    'filename': [f'{i}.wav' for i in range(500)],\n",
    "    'label': [(i // 50) for i in range(500)],\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "d1053558",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.merge(data, classes, on=['filename'], how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "e030d2f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate accuracy\n",
    "data['agreement'] = (data['label'] == data['digit'])\n",
    "\n",
    "def kappa(row):\n",
    "    row['kappa'] = cohen_kappa_score(row['label'], row['digit'])\n",
    "    return row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "96ce8d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[~data.index.isin(np.where(np.isnan(data['digit']))[0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "de7d65d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uid</th>\n",
       "      <th>method</th>\n",
       "      <th>worker</th>\n",
       "      <th>digit</th>\n",
       "      <th>intelligibility</th>\n",
       "      <th>quality</th>\n",
       "      <th>diversity</th>\n",
       "      <th>filename</th>\n",
       "      <th>label</th>\n",
       "      <th>agreement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dc0520a487ba3b901e415c4e57030ede</td>\n",
       "      <td>diffwave-small-500k</td>\n",
       "      <td>A1NF6PELRKACS9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.wav</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dc0520a487ba3b901e415c4e57030ede</td>\n",
       "      <td>diffwave-small-500k</td>\n",
       "      <td>A26RPQDD0RQEHL</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.wav</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dc0520a487ba3b901e415c4e57030ede</td>\n",
       "      <td>diffwave-small-500k</td>\n",
       "      <td>A3CFNUD7VR2E1E</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.wav</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dc0520a487ba3b901e415c4e57030ede</td>\n",
       "      <td>diffwave-small-500k</td>\n",
       "      <td>A3CJVRJ34U70Y9</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.wav</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dc0520a487ba3b901e415c4e57030ede</td>\n",
       "      <td>diffwave-small-500k</td>\n",
       "      <td>A3DU2EWFUGQCX4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.wav</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19995</th>\n",
       "      <td>4cea2df00a66dc4e21681081399f8a8f</td>\n",
       "      <td>sashimi-diffwave-snet-uni-500k</td>\n",
       "      <td>A3M3HUU77NKTES</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>99.wav</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19996</th>\n",
       "      <td>4cea2df00a66dc4e21681081399f8a8f</td>\n",
       "      <td>sashimi-diffwave-snet-uni-500k</td>\n",
       "      <td>A3QZMGTVA4VO44</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>99.wav</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19997</th>\n",
       "      <td>4cea2df00a66dc4e21681081399f8a8f</td>\n",
       "      <td>sashimi-diffwave-snet-uni-500k</td>\n",
       "      <td>AEF601SQFOSBL</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>99.wav</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19998</th>\n",
       "      <td>4cea2df00a66dc4e21681081399f8a8f</td>\n",
       "      <td>sashimi-diffwave-snet-uni-500k</td>\n",
       "      <td>AM9XH69KBK5X5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>99.wav</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19999</th>\n",
       "      <td>4cea2df00a66dc4e21681081399f8a8f</td>\n",
       "      <td>sashimi-diffwave-snet-uni-500k</td>\n",
       "      <td>AYFUVCC48YCBZ</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>99.wav</td>\n",
       "      <td>1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    uid                          method  \\\n",
       "0      dc0520a487ba3b901e415c4e57030ede             diffwave-small-500k   \n",
       "1      dc0520a487ba3b901e415c4e57030ede             diffwave-small-500k   \n",
       "2      dc0520a487ba3b901e415c4e57030ede             diffwave-small-500k   \n",
       "3      dc0520a487ba3b901e415c4e57030ede             diffwave-small-500k   \n",
       "4      dc0520a487ba3b901e415c4e57030ede             diffwave-small-500k   \n",
       "...                                 ...                             ...   \n",
       "19995  4cea2df00a66dc4e21681081399f8a8f  sashimi-diffwave-snet-uni-500k   \n",
       "19996  4cea2df00a66dc4e21681081399f8a8f  sashimi-diffwave-snet-uni-500k   \n",
       "19997  4cea2df00a66dc4e21681081399f8a8f  sashimi-diffwave-snet-uni-500k   \n",
       "19998  4cea2df00a66dc4e21681081399f8a8f  sashimi-diffwave-snet-uni-500k   \n",
       "19999  4cea2df00a66dc4e21681081399f8a8f  sashimi-diffwave-snet-uni-500k   \n",
       "\n",
       "               worker  digit  intelligibility  quality  diversity filename  \\\n",
       "0      A1NF6PELRKACS9      0                1        2          3    0.wav   \n",
       "1      A26RPQDD0RQEHL      0                1        3          3    0.wav   \n",
       "2      A3CFNUD7VR2E1E      3                1        2          4    0.wav   \n",
       "3      A3CJVRJ34U70Y9      0                1        2          4    0.wav   \n",
       "4      A3DU2EWFUGQCX4      0                2        2          4    0.wav   \n",
       "...               ...    ...              ...      ...        ...      ...   \n",
       "19995  A3M3HUU77NKTES      1                4        5          5   99.wav   \n",
       "19996  A3QZMGTVA4VO44      1                5        5          5   99.wav   \n",
       "19997   AEF601SQFOSBL      1                4        3          3   99.wav   \n",
       "19998   AM9XH69KBK5X5      1                3        3          3   99.wav   \n",
       "19999   AYFUVCC48YCBZ      1                5        3          3   99.wav   \n",
       "\n",
       "       label  agreement  \n",
       "0          0       True  \n",
       "1          0       True  \n",
       "2          0      False  \n",
       "3          0       True  \n",
       "4          0       True  \n",
       "...      ...        ...  \n",
       "19995      1       True  \n",
       "19996      1       True  \n",
       "19997      1       True  \n",
       "19998      1       True  \n",
       "19999      1       True  \n",
       "\n",
       "[20000 rows x 10 columns]"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "d3562f73",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>intelligibility</th>\n",
       "      <th>quality</th>\n",
       "      <th>diversity</th>\n",
       "      <th>agreement</th>\n",
       "      <th>kappa</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>diffwave-small-500k</th>\n",
       "      <td>1.8950</td>\n",
       "      <td>1.850</td>\n",
       "      <td>3.032</td>\n",
       "      <td>0.5014</td>\n",
       "      <td>0.446000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-snet-uni-500k</th>\n",
       "      <td>3.2916</td>\n",
       "      <td>3.076</td>\n",
       "      <td>3.260</td>\n",
       "      <td>0.8462</td>\n",
       "      <td>0.829111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-small-500k</th>\n",
       "      <td>4.0034</td>\n",
       "      <td>3.832</td>\n",
       "      <td>3.338</td>\n",
       "      <td>0.9406</td>\n",
       "      <td>0.934000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-800k</th>\n",
       "      <td>4.3292</td>\n",
       "      <td>4.200</td>\n",
       "      <td>3.284</td>\n",
       "      <td>0.9578</td>\n",
       "      <td>0.953111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                intelligibility  quality  diversity  \\\n",
       "method                                                                \n",
       "diffwave-small-500k                      1.8950    1.850      3.032   \n",
       "sashimi-diffwave-snet-uni-500k           3.2916    3.076      3.260   \n",
       "sashimi-diffwave-small-500k              4.0034    3.832      3.338   \n",
       "sashimi-diffwave-800k                    4.3292    4.200      3.284   \n",
       "\n",
       "                                agreement     kappa  \n",
       "method                                               \n",
       "diffwave-small-500k                0.5014  0.446000  \n",
       "sashimi-diffwave-snet-uni-500k     0.8462  0.829111  \n",
       "sashimi-diffwave-small-500k        0.9406  0.934000  \n",
       "sashimi-diffwave-800k              0.9578  0.953111  "
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.groupby('method').apply(kappa).groupby('method').mean().sort_values('intelligibility')[['intelligibility', 'quality', 'diversity', 'agreement', 'kappa']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "740d2b21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_stats(df, col, div=1.):\n",
    "    stats = df.groupby(['method'])[col].agg(['mean', 'count', 'std'])\n",
    "    ci95_hi = []\n",
    "    ci95_lo = []\n",
    "\n",
    "    for i in stats.index:\n",
    "        m, c, s = stats.loc[i]\n",
    "        c /= div\n",
    "        ci95_hi.append(m + 1.96*s/math.sqrt(c))\n",
    "        ci95_lo.append(m - 1.96*s/math.sqrt(c))\n",
    "\n",
    "    stats['ci95_hi'] = ci95_hi\n",
    "    stats['ci95_lo'] = ci95_lo\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "0b84cea9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>std</th>\n",
       "      <th>ci95_hi</th>\n",
       "      <th>ci95_lo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>diffwave-small-500k</th>\n",
       "      <td>1.85</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.88</td>\n",
       "      <td>1.93</td>\n",
       "      <td>1.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-800k</th>\n",
       "      <td>4.20</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.67</td>\n",
       "      <td>4.26</td>\n",
       "      <td>4.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-small-500k</th>\n",
       "      <td>3.83</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.77</td>\n",
       "      <td>3.90</td>\n",
       "      <td>3.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-snet-uni-500k</th>\n",
       "      <td>3.08</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.86</td>\n",
       "      <td>3.15</td>\n",
       "      <td>3.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                mean  count   std  ci95_hi  ci95_lo\n",
       "method                                                             \n",
       "diffwave-small-500k             1.85   5000  0.88     1.93     1.77\n",
       "sashimi-diffwave-800k           4.20   5000  0.67     4.26     4.14\n",
       "sashimi-diffwave-small-500k     3.83   5000  0.77     3.90     3.76\n",
       "sashimi-diffwave-snet-uni-500k  3.08   5000  0.86     3.15     3.00"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_stats(data, 'quality', div=10.).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "31867a1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>std</th>\n",
       "      <th>ci95_hi</th>\n",
       "      <th>ci95_lo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>diffwave-small-500k</th>\n",
       "      <td>1.90</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.12</td>\n",
       "      <td>1.93</td>\n",
       "      <td>1.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-800k</th>\n",
       "      <td>4.33</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.94</td>\n",
       "      <td>4.36</td>\n",
       "      <td>4.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-small-500k</th>\n",
       "      <td>4.00</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.08</td>\n",
       "      <td>4.03</td>\n",
       "      <td>3.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-snet-uni-500k</th>\n",
       "      <td>3.29</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.31</td>\n",
       "      <td>3.33</td>\n",
       "      <td>3.26</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                mean  count   std  ci95_hi  ci95_lo\n",
       "method                                                             \n",
       "diffwave-small-500k             1.90   5000  1.12     1.93     1.86\n",
       "sashimi-diffwave-800k           4.33   5000  0.94     4.36     4.30\n",
       "sashimi-diffwave-small-500k     4.00   5000  1.08     4.03     3.97\n",
       "sashimi-diffwave-snet-uni-500k  3.29   5000  1.31     3.33     3.26"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_stats(data, 'intelligibility').round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "a0762b85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "      <th>std</th>\n",
       "      <th>ci95_hi</th>\n",
       "      <th>ci95_lo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>diffwave-small-500k</th>\n",
       "      <td>3.03</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.17</td>\n",
       "      <td>3.13</td>\n",
       "      <td>2.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-800k</th>\n",
       "      <td>3.28</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.16</td>\n",
       "      <td>3.39</td>\n",
       "      <td>3.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-small-500k</th>\n",
       "      <td>3.34</td>\n",
       "      <td>5000</td>\n",
       "      <td>1.02</td>\n",
       "      <td>3.43</td>\n",
       "      <td>3.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sashimi-diffwave-snet-uni-500k</th>\n",
       "      <td>3.26</td>\n",
       "      <td>5000</td>\n",
       "      <td>0.86</td>\n",
       "      <td>3.34</td>\n",
       "      <td>3.18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                mean  count   std  ci95_hi  ci95_lo\n",
       "method                                                             \n",
       "diffwave-small-500k             3.03   5000  1.17     3.13     2.93\n",
       "sashimi-diffwave-800k           3.28   5000  1.16     3.39     3.18\n",
       "sashimi-diffwave-small-500k     3.34   5000  1.02     3.43     3.25\n",
       "sashimi-diffwave-snet-uni-500k  3.26   5000  0.86     3.34     3.18"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_stats(data, 'diversity', 10.).round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b37ad4a3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
